/*
 *  Copyright 2004 Blandware (http://www.blandware.com)
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.blandware.atleap.search;

import com.blandware.atleap.common.Constants;
import com.blandware.atleap.common.parsers.PlainTextExtractor;
import com.blandware.atleap.common.parsers.exception.PlainTextExtractorException;
import com.blandware.atleap.common.parsers.exception.UnsupportedMimeTypeException;
import com.blandware.atleap.common.util.ConvertUtil;
import com.blandware.atleap.common.util.DateUtil;
import com.blandware.atleap.common.util.StringUtil;
import com.blandware.atleap.model.core.*;
import com.blandware.atleap.model.core.statistics.SearchCase;
import com.blandware.atleap.search.analysis.FrenchAnalyzer;
import com.blandware.atleap.search.analysis.NorwegianAnalyzer;
import com.blandware.atleap.search.analysis.PortugueseAnalyzer;
import com.blandware.atleap.search.analysis.SpanishAnalyzer;
import com.blandware.atleap.search.filters.RolesFilter;
import com.blandware.atleap.service.core.ContentResourceManager;
import com.blandware.atleap.service.core.LayoutManager;
import com.blandware.atleap.service.core.PageManager;
import com.blandware.atleap.service.core.StatisticsManager;
import com.blandware.atleap.webapp.struts.HeritableComponentDefinition;
import com.blandware.atleap.webapp.util.core.GlobalProperties;
import com.blandware.atleap.webapp.util.core.WebappConstants;
import com.blandware.atleap.webapp.util.core.WebappUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.cz.CzechAnalyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.el.GreekAnalyzer;
import org.apache.lucene.analysis.nl.DutchAnalyzer;
import org.apache.lucene.analysis.ru.RussianAnalyzer;
import org.apache.lucene.analysis.snowball.SnowballAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.misc.ChainedFilter;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.struts.tiles.TilesUtil;
import org.springframework.context.ApplicationContext;
import org.springframework.web.context.support.WebApplicationContextUtils;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;

/**
 * <p>Manager to perform search operations</p>
 * <p><a href="SearchManager.java.html"><i>View Source</i></a></p>
 * <p/>
 *
 * @author Andrey Grebnev <a href="mailto:andrey.grebnev@blandware.com">&lt;andrey.grebnev@blandware.com&gt;</a>
 * @version $Revision: 1.45 $ $Date: 2008/02/19 12:45:06 $
 */
public class SearchManager {

	public static final String URI_FIELD = "uri";
	public static final String BODY_FIELD = "body";
	public static final String TITLE_FIELD = "title";
	public static final String SUMMARY_FIELD = "summary";
	public static final String MODIFIED_FIELD = "modified";
	public static final String LOCALE_FIELD = "locale";
	public static final String ROLES_FIELD = "roles";
	public static final String TYPE_FIELD = "type";

	public static final String PAGE_TYPE = "page";
	public static final String DOCUMENT_TYPE = "document";

	protected static final String SEARCH_DIR_KEY = "search.index.dir";

	protected static final String HTML_MIMETYPE = "text/html";

	protected static final int RESULT_FRAGMENT_SIZE = 60;
	protected static final int RESULT_FRAGMENT_NUMBER = 3;
	protected static final String RESULT_FRAGMENT_DELIMITER = "...";

	protected static final String INSTANCE_KEY = "com.blandware.atleap.search.SearchManager.INSTANCE";

	protected transient final Log log = LogFactory.getLog(SearchManager.class);

	protected ServletContext servletContext = null;
    protected File indexDir = null;

    /**
     * Flag to indicate whether the initialize finction was called for this SearchManager instance or not
     */
    protected boolean initialized = false;

    /**
     * Index reader used for search facilities
     */
    protected IndexReader searchIndexReader = null;

    /**
     * Index reader used for unindex facilities
     */
    protected IndexReader unindexIndexReader = null;

    /**
     * Index searcher
     */
    protected IndexSearcher indexSearcher = null;

    /**
     * Map of analyzers where key is locale
     */
    protected Map analyzers = null;

    /**
     * Application context
     */
	protected ApplicationContext applicationCtx = null;

    /**
     * This map contains Filters by locale key
     */
    protected HashMap localeFilters = new HashMap();

    /**
     * This map contains Filters by comma separated roles string
     */
    protected HashMap rolesFilters = new HashMap();


	/**
	 * Returns instance of SearchManager
	 *
	 * @param servletContext servlet context
	 * @return Instance of SearchManager
	 */
	public static SearchManager getInstance(ServletContext servletContext) {
		SearchManager ourInstance = (SearchManager) servletContext.getAttribute(INSTANCE_KEY);
		if ( ourInstance == null ) {
			ourInstance = new SearchManager(servletContext);
			servletContext.setAttribute(INSTANCE_KEY, ourInstance);
		}
		return ourInstance;
	}

	/**
	 * Private constructor
	 *
	 * @param servletContext servlet context
	 */
	protected SearchManager(ServletContext servletContext) {
		applicationCtx = WebApplicationContextUtils.getRequiredWebApplicationContext(servletContext);

        analyzers = new HashMap();

		String indexDirPath = GlobalProperties.getInstance(servletContext).getString(SEARCH_DIR_KEY);
		if ( indexDirPath == null || indexDirPath.length() == 0 ) {
			String error = "The property '" + SEARCH_DIR_KEY + "' must be specified";
			if ( log.isErrorEnabled() ) {
				log.error(error);
			}
			throw new RuntimeException(error);
		}

        //make lock dir if it does not exist
        File lockDir = new File(FSDirectory.LOCK_DIR);
        if (!lockDir.exists()) {
            boolean created = true;
            try {
                created = lockDir.mkdirs();
            } catch ( Exception ex ) {
                created = false;
            }
            if ( !created ) {
                if ( log.isErrorEnabled() ) {
                    log.error("Cannot create dir " + lockDir.getAbsolutePath() + " for search index lock files");
                }
            }
        } else {
            if (!lockDir.canWrite() || !lockDir.canRead()) {
                if ( log.isErrorEnabled() ) {
                    log.error("The system has not permissions to write into " + lockDir.getAbsolutePath());
                }
            }
        }

        //create or unlock index dir
		indexDir = new File(indexDirPath);
		this.servletContext = servletContext;
		if ( indexDir.exists() ) {
			try {
				Directory index = FSDirectory.getDirectory(indexDir, false);
				if ( IndexReader.isLocked(indexDir.getAbsolutePath()) ) {
					IndexReader.unlock(index);
				}
			} catch ( IOException ex ) {
                if ( log.isErrorEnabled() ) {
                    log.error("Cannot remove lock from search index " + indexDir.getAbsolutePath());
                }
			}
		} else {
			boolean created = true;
			try {
				created = indexDir.mkdirs();
			} catch ( Exception ex ) {
				created = false;
			}
			if ( !created ) {
				if ( log.isErrorEnabled() ) {
					log.error("Cannot create dir " + indexDir.getAbsolutePath() + " for search index");
				}
			}

            if ( log.isInfoEnabled() ) {
                log.info("Search manager initialized");
            }
        }

        //recreate index
        IndexWriter indexWriter = null;
        try {
            indexWriter = getIndexWriter("", true);
        } finally {
            closeIndexWriter(indexWriter);
        }
	}

	// P A G E S

	/**
	 * Indexes any child of <code>Page</code> class
	 *
	 * @param page      page to index
	 * @param request   request
	 */
	public synchronized void indexPage(Page page, HttpServletRequest request) {
		initialize(request);
		if ( page instanceof ContentPage ) {
			indexContentPage((ContentPage) page, request);
		} else if (page instanceof ActionPage) {
            indexActionPage((ActionPage) page, request);
        } else {
            String roles = WebappUtil.rolesToString(page.getRoles());
			indexPageFields(page.getContentFields(), page, roles);
		}
	}

	/**
	 * Reindexes any child of <code>Page</code> class
	 *
	 * @param page      page to reindex
	 * @param request   request
	 */
	public synchronized void reIndexPage(Page page, HttpServletRequest request) {
		unIndexPage(page.getUri(), request);
		indexPage(page, request);
	}

	/**
	 * Unindexes page by specified uri
	 *
	 * @param uri       page uri by which to remove from index
     * @param request   request
	 */
	public synchronized void unIndexPage(String uri, HttpServletRequest request) {
		initialize(request);
		unIndex(uri);
	}

	/**
	 * Indexes content page including fields from its layout and its parent layouts
	 *
	 * @param contentPage   content page to index
	 * @param request       request
	 */
	protected synchronized void indexContentPage(ContentPage contentPage, HttpServletRequest request) {
		Collection allFields = null;
		List layouts = new ArrayList();
		LayoutManager layoutManager = (LayoutManager) applicationCtx.getBean(Constants.LAYOUT_MANAGER_BEAN);
		String tmpDefinition = contentPage.getLayout().getDefinition();
		try {
			do {
				Layout layout = layoutManager.findLayoutByDefinition(tmpDefinition);
				if ( layout != null ) {
					layouts.add(layout);
				}
				tmpDefinition = ((HeritableComponentDefinition) TilesUtil.getDefinition(tmpDefinition, request, servletContext)).getExtends();
			} while ( tmpDefinition != null );

		} catch ( Exception ex ) {
			if ( log.isErrorEnabled() ) {
				log.error("Cannot traverse definitions", ex);
			}
		}
		Collections.reverse(layouts);
		for ( int i = 0; i < layouts.size(); i++ ) {
			Layout layout = (Layout) layouts.get(i);
			allFields = WebappUtil.joinFields(allFields, layout.getContentFields());
		}
		allFields = WebappUtil.joinFields(allFields, contentPage.getContentFields());

		indexPageFields(allFields, contentPage, WebappUtil.rolesToString(contentPage.getRoles()));
	}

    /**
     * Indexes action page
     * @param actionPage    action page to index
     * @param request       request
     */
    protected synchronized void indexActionPage(ActionPage actionPage, HttpServletRequest request) {
        String roles = ConvertUtil.convertListToString(WebappUtil.getAPRoleNamesAsList(actionPage.getUri(), request), ",");
        indexPageFields(actionPage.getContentFields(), actionPage, roles);
    }

	// D O C U M E N T   R E S O U R C E

	/**
	 * Indexes document resource
	 *
	 * @param resourceDocument  document to index
	 * @param request           request
	 */
	public synchronized void indexDocument(ContentDocument resourceDocument, HttpServletRequest request) {
		initialize(request);
		byte[] resourceData = resourceDocument.getResourceData().getData();
		String language = resourceDocument.getContentLocale().getIdentifier();

		String plainText = null;
		try {
			plainText = new PlainTextExtractor().extract(new ByteArrayInputStream(resourceData), resourceDocument.getMimeType(), resourceDocument.getCharset());
		} catch ( UnsupportedMimeTypeException ex ) {
			if ( log.isErrorEnabled() ) {
				log.error("Unsupported mime type " + resourceDocument.getMimeType(), ex);
			}
		} catch ( PlainTextExtractorException ex ) {
			if ( log.isErrorEnabled() ) {
				log.error("Cannot parse resource document with mimetype " + resourceDocument.getMimeType(), ex);
			}
		}

		if ( plainText != null ) {
			Integer summarySize = GlobalProperties.getInstance(request.getSession().getServletContext()).getInteger(WebappConstants.DOCUMENT_SUMMARY_SIZE_KEY, new Integer(400));
			String summary = plainText.substring(0, Math.min(plainText.length(), summarySize.intValue()));

			Document document = new Document();
			document.add(new Field(URI_FIELD, resourceDocument.getUri(), Field.Store.YES, Field.Index.UN_TOKENIZED));
			document.add(new Field(BODY_FIELD, plainText, Field.Store.NO, Field.Index.TOKENIZED));
			document.add(new Field(MODIFIED_FIELD, DateTools.dateToString(resourceDocument.getLastUpdatedDatetime(), DateTools.Resolution.SECOND), Field.Store.YES, Field.Index.UN_TOKENIZED));
			document.add(new Field(TYPE_FIELD, DOCUMENT_TYPE, Field.Store.YES, Field.Index.UN_TOKENIZED));
			document.add(new Field(LOCALE_FIELD, language, Field.Store.YES, Field.Index.UN_TOKENIZED));
			document.add(new Field(SUMMARY_FIELD, summary, Field.Store.YES, Field.Index.NO));
			String roles = WebappUtil.rolesToString(resourceDocument.getRoles());
			if ( roles != null ) {
				document.add(new Field(ROLES_FIELD, roles, Field.Store.YES, Field.Index.NO));
			}

			String description = (String) resourceDocument.getDescription().get(language);
			if ( description != null && description.trim().length() > 0 ) {
				document.add(new Field(TITLE_FIELD, description, Field.Store.YES, Field.Index.TOKENIZED));
			}

			IndexWriter indexWriter = null;
			try {
				indexWriter = getIndexWriter(language);

				if ( log.isDebugEnabled() ) {
					log.debug("Adding into index resource document: " + document + " for locale " + language);
				}
				//add document
				indexWriter.addDocument(document);
			} catch ( IOException ex ) {
				if ( log.isErrorEnabled() ) {
					log.error("Cannot add resource document into index ", ex);
				}
			} finally {
				closeIndexWriter(indexWriter);
			}
		}
	}

	/**
	 * Reindexes document resource
	 *
	 * @param document  document to reindex
	 * @param request   request
	 */
	public synchronized void reIndexDocument(ContentDocument document, HttpServletRequest request) {
		unIndexDocument(document.getUri(), request);
		indexDocument(document, request);
	}

	/**
	 * Unindexes document resource by specified uri
	 *
	 * @param uri   resource URI by which to remove from index
     * @param request request
	 */
	public synchronized void unIndexDocument(String uri, HttpServletRequest request) {
		initialize(request);
		unIndex(uri);
	}

	// C O M M O N   M E T H O D S

	/**
	 * Indexes content of the fields, after it they will be available to search
	 *
	 * @param fields array of the <code>ContentField</code> objects
	 * @param page   Page object
     * @param roles comma separated list of roles
	 */
	protected synchronized void indexPageFields(Collection fields, Page page, String roles) {
		Map documents = new HashMap();
		Map documentModifications = new HashMap();

		if ( log.isDebugEnabled() ) {
			log.debug("Indexing " + fields.size() + " fields for uri '" + page.getUri() + "'");
		}

		//iterate list of ContentFields
		for ( Iterator iterator = fields.iterator(); iterator.hasNext(); ) {
			ContentField contentField = (ContentField) iterator.next();
			List contents = contentField.getContentFieldValues();
			String identifier = contentField.getIdentifier();

			//iterate list of ContentFieldValue
			for ( int j = 0; j < contents.size(); j++ ) {
				ContentFieldValue contentFieldValue = (ContentFieldValue) contents.get(j);
				ContentLocale contentLocale = contentFieldValue.getContentLocale();
				String locale = contentLocale.getIdentifier();

				//convert to string
				String value = null;
				if ( contentField.getType() == ContentField.LINE_TYPE ) {
					value = contentFieldValue.getSimpleValue();
				} else {
					value = ConvertUtil.convertToString(contentFieldValue.getValue(), Constants.DEFAULT_ENCODING);
				}
				if ( value == null ) {
					continue;
				}

				//looking up document for this locale in the map
				Document document = (Document) documents.get(locale);
				//create if does not exist and add to the map
				if ( document == null ) {
					document = new Document();
					documents.put(locale, document);
					document.add(new Field(URI_FIELD, page.getUri(), Field.Store.YES, Field.Index.UN_TOKENIZED));

    				document.add(new Field(TYPE_FIELD, PAGE_TYPE, Field.Store.YES, Field.Index.UN_TOKENIZED));
                    document.add(new Field(ROLES_FIELD, roles, Field.Store.YES, Field.Index.NO));
				}

				//put into map maximum date of modification
				Date modificationDate = (Date) documentModifications.get(locale);
				if ( modificationDate == null ) {
					documentModifications.put(locale, (Date) contentFieldValue.getLastUpdatedDatetime());
					if ( log.isDebugEnabled() ) {
						log.debug("Add into documentModification locale-" + locale + ", date-" + contentFieldValue.getLastUpdatedDatetime());
					}
				} else if ( modificationDate.before((Date) contentFieldValue.getLastUpdatedDatetime()) ) {
					documentModifications.put(locale, (Date) contentFieldValue.getLastUpdatedDatetime());
					if ( log.isDebugEnabled() ) {
						log.debug("Update in documentModification locale-" + locale + ", date-" + contentFieldValue.getLastUpdatedDatetime());
					}
				}

				//add fields to document
				if ( identifier.equalsIgnoreCase(ContentField.TITLE_IDENTIFIER) ) {
					document.add(new Field(BODY_FIELD, value, Field.Store.NO, Field.Index.TOKENIZED));
					document.add(new Field(TITLE_FIELD, value, Field.Store.YES, Field.Index.TOKENIZED));
				} else if ( identifier.equalsIgnoreCase(ContentField.KEYWORDS_IDENTIFIER) ) {
					String[] keywords = value.split(",");
					for ( int k = 0; k < keywords.length; k++ ) {
						//neither stored not tokenized, but indexed
						document.add(new Field(BODY_FIELD, keywords[k], Field.Store.NO, Field.Index.UN_TOKENIZED));
					}
				} else if ( identifier.equalsIgnoreCase(ContentField.DESCRIPTION_IDENTIFIER) ) {
					document.add(new Field(BODY_FIELD, value, Field.Store.NO, Field.Index.TOKENIZED));
				} else {
					if ( ContentField.HTML_TYPE == contentField.getType() ) {
						try {
							String plainText = new PlainTextExtractor().extract(value, HTML_MIMETYPE);
							document.add(new Field(BODY_FIELD, plainText, Field.Store.NO, Field.Index.TOKENIZED));
						} catch ( Exception ex ) {
							if ( log.isErrorEnabled() ) {
								log.error("The content cannot be parsed ", ex);
							}
						}
					} else {
						document.add(new Field(BODY_FIELD, value, Field.Store.NO, Field.Index.TOKENIZED));
					}
				}
			} //end of FeildContent iteration
		} //end of Feild iteration

		//iterate locales and index document for each locale
		Iterator localesIterator = documents.keySet().iterator();
		while ( localesIterator.hasNext() ) {
			String locale = (String) localesIterator.next();
			Document document = (Document) documents.get(locale);
			IndexWriter indexWriter = null;
			try {
				indexWriter = getIndexWriter(locale);
				//add modification
				Date modificationDate = (Date) documentModifications.get(locale);
				if ( modificationDate != null ) {
					document.add(new Field(MODIFIED_FIELD, DateTools.dateToString(modificationDate, DateTools.Resolution.SECOND), Field.Store.YES, Field.Index.UN_TOKENIZED));
				}

				//add locale
				document.add(new Field(LOCALE_FIELD, locale, Field.Store.YES, Field.Index.UN_TOKENIZED));

				if ( log.isDebugEnabled() ) {
					log.debug("Adding into index document: " + document + " for locale " + locale);
				}

				//add document
				indexWriter.addDocument(document);
			} catch ( IOException ex ) {
				if ( log.isErrorEnabled() ) {
					log.error("Cannot add document into index ", ex);
				}
			} finally {
				closeIndexWriter(indexWriter);
			}
		} //end locale iterator
	}

	/**
	 * Removes documents with specified uri from index
	 *
	 * @param uri module-relative uri
	 */
	protected synchronized void unIndex(String uri) {
        loadUnindexIndexReader();
		Term term = new Term(URI_FIELD, uri);
		if ( log.isDebugEnabled() ) {
			log.debug("Try to unindex documents using term: " + term.toString());
		}
		try {
			int number = unindexIndexReader.deleteDocuments(term);
			if ( log.isDebugEnabled() ) {
				log.debug("Unindexed " + number + " documents using uri: " + uri);
			}
		} catch ( IOException ex ) {
			if ( log.isErrorEnabled() ) {
				log.error("Cannot remove documents from index ", ex);
			}
		}
	}

	/**
	 * Unindexes all documents by locale
	 *
	 * @param locale locale for which documents must be unindexed
     * @param request request
	 */
	public synchronized void unIndexByLocale(String locale, HttpServletRequest request) {
		initialize(request);
        loadUnindexIndexReader();
		Term term = new Term(LOCALE_FIELD, locale);
		if ( log.isDebugEnabled() ) {
			log.debug("Try to unindex documents using term: " + term.toString());
		}
		try {
			int number = unindexIndexReader.deleteDocuments(term);
			if ( log.isDebugEnabled() ) {
				log.debug("Unindexed " + number + " documents using locale: " + locale);
			}
		} catch ( IOException ex ) {
			if ( log.isErrorEnabled() ) {
				log.error("Cannot remove documents from index ", ex);
			}
		}
	}


	/**
	 * Searches by query string and hightlights result only for current page
	 *
     * @param hits hits list from previous page or null
	 * @param criteria query string
	 * @param locale current language
     * @param allLanguages if <code>true</code> it means the search will be performed on all languages but using alanyzer on specified locale
     *        otherwise the search will be performed on only on specified one
	 * @param days   if more that zero show result only for last N days, otherwise all
     * @param roles comma separated list of roles of current user to restict results by roles
	 * @param offset   offset for pagination
	 * @param pageSize pageSize for pagination
	 * @param request  http request
	 * @return list of <code>Hit</code>
	 * @throws ParseException if there is an error during parsing of the criteria
	 * @throws IOException    if there is a problem with writing or reading index
	 */
	public synchronized List search(List hits, String criteria, String locale, boolean allLanguages, int days, String roles, int offset, int pageSize, HttpServletRequest request)
	        throws ParseException, IOException, java.text.ParseException {
		initialize(request);

		loadIndexSearcher();

		QueryParser queryParser = new QueryParser(BODY_FIELD, getAnalyzer(locale));
		Query query = queryParser.parse(criteria);

		if ( hits == null ) {
			if ( log.isDebugEnabled() ) {
				log.debug("Searching for: " + query.toString(BODY_FIELD));
			}

            List filters = new ArrayList();
            //locale
            if (!allLanguages) {
                QueryFilter localeFilter = (QueryFilter)localeFilters.get(locale);
                if (localeFilter == null) {
                    localeFilter = new QueryFilter(new TermQuery(new Term(LOCALE_FIELD, locale)));
                    localeFilters.put(locale, localeFilter);
                }
                filters.add(localeFilter);
            }

            //roles
            filters.add(new CachingWrapperFilter(new RolesFilter(roles)));

            //time period in days
            if ( days > 0 ) {
                Calendar calendar = Calendar.getInstance();
                calendar.add(Calendar.DATE, -days);
                filters.add(new CachingWrapperFilter(RangeFilter.More(SearchManager.MODIFIED_FIELD, DateTools.dateToString(calendar.getTime(), DateTools.Resolution.SECOND))));
            }

            ChainedFilter chainedFilter = new ChainedFilter((Filter[])filters.toArray(new Filter[filters.size()]), ChainedFilter.AND);

			hits = prepareHitsList(indexSearcher.search(query, chainedFilter), locale, request);

            if (GlobalProperties.getInstance(servletContext).getBoolean(WebappConstants.STATISTICS_ENABLED_KEY, true).booleanValue()) {
                registerSearchCase(criteria);
            }
        }

        int limit = (offset + pageSize) > hits.size() ? hits.size() : (offset + pageSize);

		//preparing search results only for requested page
		for ( int i = offset; i < limit; i++ ) {
			Hit hit = (Hit) hits.get(i);

            //in order to do not do double work
            if (hit.getHighlightedText() != null)
                continue;

			String type = hit.getType();
			//for any multi-term queries to work (prefix, wildcard, range,fuzzy etc) you must use a rewritten query!
			query = query.rewrite(searchIndexReader);
			Highlighter highlighter = new Highlighter(new QueryScorer(query));
			highlighter.setTextFragmenter(new SimpleFragmenter(RESULT_FRAGMENT_SIZE));

            String documentLocale = hit.getLocale();
            Analyzer documentAnalyzer = getAnalyzer(documentLocale);

			if ( !DOCUMENT_TYPE.equalsIgnoreCase(type) ) {

				PageManager pageManager = (PageManager) applicationCtx.getBean(Constants.PAGE_MANAGER_BEAN);
				Page page = pageManager.findPageByUri(hit.getOriginalUri());
				if (page == null) {
					if(log.isWarnEnabled()) {
						log.warn("Search index is not synchronized with database. Page with uri=" + hit.getOriginalUri() + " was not found");
					}
					continue;
				}
				Collection fields = page.getContentFields();

				if ( fields != null && fields.size() > 0 ) {
					StringBuffer highlightedText = new StringBuffer("");
					for ( Iterator iterator = fields.iterator(); iterator.hasNext(); ) {
						ContentField contentField = (ContentField) iterator.next();
						//keywords cannot be shown in the result
						if ( !contentField.getIdentifier().equalsIgnoreCase(ContentField.KEYWORDS_IDENTIFIER) ) {
							List fieldValues = contentField.getContentFieldValues();
							for ( int j = 0; j < fieldValues.size(); j++ ) {
								ContentFieldValue contentFieldValue = (ContentFieldValue) fieldValues.get(j);
								if ( contentFieldValue.getContentLocale().getIdentifier().equalsIgnoreCase(documentLocale) ) {
									String content = null;
									if ( contentField.getType() == ContentField.LINE_TYPE ) {
										content = contentFieldValue.getSimpleValue();
									} else {
										content = ConvertUtil.convertToString(contentFieldValue.getValue(), Constants.DEFAULT_ENCODING);
									}
                                    if (content == null || content.trim().length() == 0)
                                        continue;
									if ( contentField.getIdentifier().equalsIgnoreCase(ContentField.TITLE_IDENTIFIER) ) {
                                        content = StringUtil.htmlEncode(content);
										TokenStream tokenStream = documentAnalyzer.tokenStream(null, new StringReader(content));
										String fragment = highlighter.getBestFragment(tokenStream, content);
										if ( fragment != null && fragment.length() > 0 ) {
											hit.setTitle(fragment);
										}
									} else {
										if ( ContentField.HTML_TYPE == contentField.getType() ) {
											try {
												content = new PlainTextExtractor().extract(content, HTML_MIMETYPE);
											} catch ( Exception ex ) {
												if ( log.isErrorEnabled() ) {
													log.error("The content cannot be parsed ", ex);
												}
											}
										} else {
                                            content = StringUtil.htmlEncode(content);
                                        }
										TokenStream tokenStream = documentAnalyzer.tokenStream(null, new StringReader(content));
										String fragment = highlighter.getBestFragments(tokenStream, content, RESULT_FRAGMENT_NUMBER, RESULT_FRAGMENT_DELIMITER);
										if ( fragment != null && fragment.length() > 0 ) {
											highlightedText.append(fragment).append(RESULT_FRAGMENT_DELIMITER);
										}
									}
								}
							}
						}
					}

					String highlightedTextString = highlightedText.toString();
					if ( highlightedTextString != null && highlightedTextString.length() > 0 ) {
						if ( log.isDebugEnabled() ) {
							log.debug("Highlighted text: " + highlightedTextString);
						}
						hit.setHighlightedText(highlightedTextString);
					}
				}
			} else /* DOCUMENT type */ {

				String summary = hit.getSummary();
				if ( summary != null ) {
                    summary = StringUtil.htmlEncode(summary);
					TokenStream tokenStream = documentAnalyzer.tokenStream(null, new StringReader(summary));
					String fragment = highlighter.getBestFragments(tokenStream, summary, RESULT_FRAGMENT_NUMBER, RESULT_FRAGMENT_DELIMITER);
					if ( fragment != null && fragment.length() > 0 ) {
						if ( log.isDebugEnabled() ) {
							log.debug("Highlighted text: " + fragment);
						}
						hit.setHighlightedText(fragment);
					} else {
						if ( log.isDebugEnabled() ) {
							log.debug("No words found in summary, so using unhighlighted summary as 'highlighted text': " + summary);
						}
						hit.setHighlightedText(summary);
                    }
				}

			}
		}

		return hits;
	}

    /**
     * Registers search case putting info about it into database.
     *
     * @param query search query
     */
    protected void registerSearchCase(String query) {
        StatisticsManager statisticsManager = (StatisticsManager) applicationCtx.getBean(Constants.STATISTICS_MANAGER_BEAN);
        SearchCase searchCase = new SearchCase();
        searchCase.setQuery(query);
        searchCase.setDate(new Date());
        statisticsManager.createSearchCase(searchCase);
    }

    /**
	 * This method prepares list of <code>Hit</code> objects. It filters list by roles and setting up general properties
	 *
	 * @param hits      hit list to prepare
	 * @param request   request
	 * @return prepared and filtered list of <code>Hit</code> objects
	 */
	protected synchronized List prepareHitsList(Hits hits, String locale, HttpServletRequest request) throws IOException, java.text.ParseException {
		if ( log.isDebugEnabled() ) {
			log.debug("Found " + hits.length() + " hits");
		}

		List result = new ArrayList(hits.length());
		for ( int i = 0; i < hits.length(); i++ ) {
			Document doc = hits.doc(i);
			String type = doc.get(TYPE_FIELD);
			String uri = doc.get(SearchManager.URI_FIELD);
            String documentLocale = doc.get(LOCALE_FIELD);
			if ( log.isDebugEnabled() ) {
				log.debug("Checking for roles hit with uri=" + uri + ", type=" + type);
			}

			Hit hit = new Hit();

			//original uri
			hit.setOriginalUri(uri);

            //set document locale
            hit.setLocale(documentLocale);

			//uri
			String fullUri;
			if ( DOCUMENT_TYPE.equalsIgnoreCase(type) ) {
				fullUri = request.getContextPath() + uri;
			} else {
				fullUri = WebappUtil.getActionMappingURL(uri, null, request, WebappConstants.URL_TYPE_DOMAIN_RELATIVE, documentLocale);
			}
			hit.setUri(fullUri);

			//type
			hit.setType(type);

			//title
			String title = doc.get(SearchManager.TITLE_FIELD);
			if ( title == null || title.trim().length() == 0 ) {
				title = fullUri;
			}
			hit.setTitle(title);

			//modified date
			Date modifiedDate = DateTools.stringToDate(doc.get(SearchManager.MODIFIED_FIELD));
			String date = DateUtil.formatDateTime(modifiedDate, new Locale(locale));
			hit.setModified(date);

			//setting score
			int score = Math.round(hits.score(i) * 100.0F);
			hit.setScore(score);

			//summary (used only by document now)
			hit.setSummary(doc.get(SUMMARY_FIELD));

			result.add(hit);
		}
		return result;
	}

	// U T I L   M E T H O D S

    /**
     * Loads index searcher
     */
    protected synchronized void loadIndexSearcher() {
        loadSearchIndexReader();
        indexSearcher = new IndexSearcher(searchIndexReader);
    }


    /**
     * Closes indexSearcher
     */
    protected synchronized void closeIndexSearcher() {
        try {
            if ( indexSearcher != null ) {
                indexSearcher.close();
                indexSearcher = null;
            }
        } catch ( IOException ex ) {
            //do nothing
        }
    }


    /**
     * Loads index reader for search
     */
    protected synchronized void loadSearchIndexReader() {
        closeUnindexIndexReader();
        if (searchIndexReader == null)
            searchIndexReader = getIndexReader();
    }


    /**
     * Closes indexReader for search
     */
    protected synchronized void closeSearchIndexReader() {
        closeIndexSearcher();
        try {
            if ( searchIndexReader != null ) {
                searchIndexReader.close();
                searchIndexReader = null;
            }
        } catch ( IOException ex ) {
            //do nothing
        }
    }

    /**
     * Loads index reader for unindex
     */
    protected synchronized void loadUnindexIndexReader() {
        closeSearchIndexReader();
        if (unindexIndexReader == null)
            unindexIndexReader = getIndexReader();
    }

    /**
     * Closes indexReader for unindex
     */
    protected synchronized void closeUnindexIndexReader() {
        try {
            if ( unindexIndexReader != null ) {
                unindexIndexReader.close();
                unindexIndexReader = null;
                optimizeIndex();
            }
        } catch ( IOException ex ) {
            //do nothing
        }
    }

    /**
     * Gets index reader
     */
    protected synchronized IndexReader getIndexReader() {
        IndexReader indexReader = null;
        try {
            indexReader = IndexReader.open(indexDir);
        } catch ( IOException ex ) {
            String error = "Cannot open index for read in " + indexDir.getAbsolutePath()
                    + File.pathSeparatorChar
                    + indexDir.getName();
            if ( log.isErrorEnabled() ) {
                log.error(error, ex);
            }
            throw new RuntimeException(error, ex);
        }
        return indexReader;
    }


	/**
	 * Loads index writer
	 *
	 * @param locale language code
     * @param create true to create the index or overwrite the existing one; false to append to the existing index
	 */
	protected synchronized IndexWriter getIndexWriter(String locale, boolean create) {
        if (!create) {
            closeSearchIndexReader();
            closeUnindexIndexReader();
        }

        IndexWriter indexWriter = null;
		try {
			indexWriter = new IndexWriter(indexDir, getAnalyzer(locale), create);
		} catch ( IOException ex ) {
			String error = "Cannot open index for write in " + indexDir.getAbsolutePath();
			if ( log.isErrorEnabled() ) {
				log.error(error, ex);
			}
			throw new RuntimeException(error, ex);
		}
        return indexWriter;
	}

    /**
     * Loads index writer, does not create/overwrite index
     *
     * @param locale language code
     */
    protected synchronized IndexWriter getIndexWriter(String locale) {
        return getIndexWriter(locale, false);
    }


	/**
	 * Finalizes indexWriter
	 */
	protected synchronized void closeIndexWriter(IndexWriter indexWriter) {
		try {
			if ( indexWriter != null ) {
				indexWriter.optimize();
				indexWriter.close();
				indexWriter = null;
			}
		} catch ( IOException ex ) {
			//do nothing
		}
	}

	/**
	 * Optimizes index
	 */
	protected synchronized void optimizeIndex() {
        IndexWriter indexWriter = null;
        try {
            indexWriter = getIndexWriter("");
        } finally {
            closeIndexWriter(indexWriter);
        }
	}

	/**
	 * Gets Analyzer for particular locale
	 *
	 * @param locale language code
	 * @return Analyzer
	 */
	protected Analyzer getAnalyzer(String locale) {
        Analyzer analyzer = (Analyzer)analyzers.get(locale);
        if (analyzer != null)
            return analyzer;

		//TODO Need to add stop words for some languages and maybe some other languages
		if ( "en".equalsIgnoreCase(locale) ) {
			analyzer = new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS);
		} else if ( "ru".equalsIgnoreCase(locale) ) {
			analyzer = new RussianAnalyzer();
		} else if ( "es".equalsIgnoreCase(locale) ) {
			analyzer = new SpanishAnalyzer();
		} else if ( "de".equalsIgnoreCase(locale) ) {
			analyzer = new GermanAnalyzer();
		} else if ( "pt".equalsIgnoreCase(locale) ) {
			analyzer = new PortugueseAnalyzer();
		} else if ( "da".equalsIgnoreCase(locale) ) {
			analyzer = new SnowballAnalyzer("Danish");
		} else if ( "fi".equalsIgnoreCase(locale) ) {
			analyzer = new SnowballAnalyzer("Finnish");
		} else if ( "fr".equalsIgnoreCase(locale) ) {
            // using patched analyzer to workaround bug in standard one
            analyzer = new FrenchAnalyzer();
		} else if ( "it".equalsIgnoreCase(locale) ) {
			analyzer = new SnowballAnalyzer("Italian");
		} else if ( "no".equalsIgnoreCase(locale) ) {
//			analyzer = new SnowballAnalyzer("Norwegian");
			analyzer = new NorwegianAnalyzer();
		} else if ( "sv".equalsIgnoreCase(locale) ) {
			analyzer = new SnowballAnalyzer("Swedish");
		} else if ( "nl".equalsIgnoreCase(locale) ) {
			analyzer = new DutchAnalyzer();
        } else if ( "ja".equalsIgnoreCase(locale) ) {
            analyzer = new CJKAnalyzer();
        } else if ( "ko".equalsIgnoreCase(locale) ) {
            analyzer = new CJKAnalyzer();
//        } else if ( "zh".equalsIgnoreCase(locale) ) {
//            analyzer = new ChineseAnalyzer();
        } else if ( "zh".equalsIgnoreCase(locale) ) {
            analyzer = new CJKAnalyzer();
        } else if ( "cs".equalsIgnoreCase(locale) ) {
            analyzer = new CzechAnalyzer();
        } else if ( "el".equalsIgnoreCase(locale) ) {
            analyzer = new GreekAnalyzer();
        } else {
            analyzer = new StandardAnalyzer();
        }

        analyzers.put(locale, analyzer);

		return analyzer;
	}


    /**
     * Initializes empty index, in our case indexes all initial pages and resources
     * @param request
     */
    protected synchronized void initialize(HttpServletRequest request) {
        if (!initialized) {
            initialized = true;
            indexAll(request);
        }
    }

	/**
	 * Closes searcher and reader
	 *
	 * @throws Throwable
	 */
	protected void finalize() throws Throwable {
		closeSearchIndexReader();
        closeUnindexIndexReader();
        super.finalize();
	}


	/**
	 * Reindexes all entities.
	 */
	public synchronized void reIndexAll(HttpServletRequest request) {
        PageManager pageManager = (PageManager) applicationCtx.getBean(Constants.PAGE_MANAGER_BEAN);
        Collection pages = pageManager.listPagesFetching();

        ContentResourceManager resourceManager = (ContentResourceManager) applicationCtx.getBean(Constants.CONTENT_RESOURCE_MANAGER_BEAN);
        Collection resources = resourceManager.listContentDocuments(null);

        //unindex
        for ( Iterator iterator = pages.iterator(); iterator.hasNext(); ) {
            Page page = (Page) iterator.next();
            unIndexPage(page.getUri(), request);
        }
        for ( Iterator iterator = resources.iterator(); iterator.hasNext(); ) {
            ContentDocument document = (ContentDocument) iterator.next();
            unIndexDocument(document.getUri(), request);
        }

        //index
        for ( Iterator iterator = pages.iterator(); iterator.hasNext(); ) {
            Page page = (Page) iterator.next();
            indexPage(page, request);
        }
        for ( Iterator iterator = resources.iterator(); iterator.hasNext(); ) {
            ContentDocument document = (ContentDocument) iterator.next();
            indexDocument(document, request);
        }

        if (log.isInfoEnabled()) {
            log.info("All pages and resources index rebuilt");
        }
    }

    /**
     * Indexes all entities.
     *
     * @param request request which is being processed
     */
    public synchronized void indexAll(HttpServletRequest request) {
        PageManager pageManager = (PageManager) applicationCtx.getBean(Constants.PAGE_MANAGER_BEAN);
        //reindex pages
        Collection pages = pageManager.listPagesFetching();
        for ( Iterator iterator = pages.iterator(); iterator.hasNext(); ) {
            Page page = (Page) iterator.next();
            indexPage(page, request);
        }

        // documents
        ContentResourceManager resourceManager = (ContentResourceManager) applicationCtx.getBean(Constants.CONTENT_RESOURCE_MANAGER_BEAN);
        Collection resources = resourceManager.listContentDocuments(null);
        for ( Iterator iterator = resources.iterator(); iterator.hasNext(); ) {
            ContentDocument document = (ContentDocument) iterator.next();
            indexDocument(document, request);
        }
    }


}
